/**
 * 
 */
package com.exceptionhandler.service.crawl;

import java.io.File;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.neocrawler.entity.WebContent;

import com.exceptionhandler.entity.OpenSourceProject;

/**
 * @author Neo
 * 
 */
public class SpringForumCrawler extends ExceptionCrawler implements
		IExceptionCrawler {
	// ========== Required!!!! ========== //
	private static final String PROJECT_NAME = "Spring";
	private static final String PARSER_CLASS_NAME = SpringForumCrawler.class
			.getName();
	private static String SEED_URLS = "http://forum.springsource.org/"
			+ OpenSourceProject.SPLITER
			+ "http://forum.springsource.org/forumdisplay.php?f=27";

	// ========== Optional ========== //
	private static final String STORE_PATH = BASE_STORE_PATH + PROJECT_NAME
			+ "\\";
	private static final boolean USE_PROXY_OR_NOT = true;
	private static final int THREAD_NUM = 6;
	private static final String PROXY_URL = "192.168.4.190";
	private static final int PROXY_PORT = 88;

	public SpringForumCrawler() {
		super(PROJECT_NAME, SEED_URLS, STORE_PATH, PARSER_CLASS_NAME,
				THREAD_NUM, USE_PROXY_OR_NOT, PROXY_URL, PROXY_PORT);
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.neocrawler.parser.IParser#getInitialSeeds()
	 */
	@Override
	public List<WebContent> getInitialSeeds() {
		return null;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * org.neocrawler.parser.IParser#parse(org.neocrawler.entity.WebContent,
	 * java.lang.String)
	 */
	@Override
	public List<WebContent> parse(WebContent webContent, String webPage) {
		List<WebContent> webContents = new ArrayList<WebContent>();
		String type = webContent.getType().toString();

		if (webPage == null) {
			log.error("传入的webPage为null！");
			return null;
		}

		if (WebContent.TYPE_SEED_PAGE.equals(type)) {
			// Parse the web page, and get all the links.
			try {

				Parser parser = Parser.createParser(webPage, "UTF-8");
				// 通过过滤器过滤出<A>标签
				NodeList nodeList = parser
						.extractAllNodesThatMatch(new NodeFilter() {
							// 实现该方法,用以过滤标签
							public boolean accept(Node node) {
								if (node instanceof LinkTag)// <A>标记
									return true;
								return false;
							}
						});
				// 打印

				// Check the links, and create different types of WebContent
				// according to the URL.
				for (int i = 0; i < nodeList.size(); i++) {
					LinkTag n = (LinkTag) nodeList.elementAt(i);
					// System.out.print(n.getLink() + " ==>> ");
					// System.out.println(n.extractLink());
					String url = n.getLink();
					if (url.contains("forumdisplay.php")) {
						WebContent w = new WebContent();
						w.setUrl(("http://forum.springsource.org/" + url)
								.replace("amp;", ""));
						w.setType(WebContent.TYPE_SEED_PAGE);
						w.setParentUrl(webContent.getUrl());
						w.setParentUuid(webContent.getUuid());
						w.setReserve02(PROJECT_NAME);
						webContents.add(w);
					} else if (url.contains("showthread.php")) {
						WebContent w = new WebContent();
						w.setUrl(("http://forum.springsource.org/" + url)
								.replace("amp;", ""));
						w.setType(TYPE_FORUM_THREAD);
						w.setParentUrl(webContent.getUrl());
						w.setParentUuid(webContent.getUuid());
						w.setReserve02(PROJECT_NAME);
						webContents.add(w);
					} else {
						// Ignore other urls.
						continue;
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}

			return webContents;
		} else if (TYPE_FORUM_THREAD.equals(type)) {
			// if forum thread, download this page.
			return null;
		} else {
			return null;
		}
	}

	public static void parserTest() {
		try {

			String webPage = FileUtils.readFileToString(new File(
					"D:\\webpage01.html"));

			Parser parser = Parser.createParser(webPage, "UTF-8");
			// 通过过滤器过滤出<A>标签
			NodeList nodeList = parser
					.extractAllNodesThatMatch(new NodeFilter() {
						// 实现该方法,用以过滤标签
						public boolean accept(Node node) {
							if (node instanceof LinkTag)// <A>标记
								return true;
							return false;
						}
					});
			// 打印
			for (int i = 0; i < nodeList.size(); i++) {
				LinkTag n = (LinkTag) nodeList.elementAt(i);
				System.out.print(n.getLink() + " ==>> ");
				System.out.println(n.extractLink());
			}
		} catch (Exception e) {
			e.printStackTrace();
		}

	}

	/**
	 * Description:
	 * 
	 * @param args
	 *            void
	 */
	public static void main(String[] args) {
		SpringForumCrawler crawler = new SpringForumCrawler();
		// crawler.start();
		crawler.restart();
	}

}
